{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "RNN_nn.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "LRCRDiskLvY5",
        "pUaw78k6LfNq",
        "E7_h-E05EhUI"
      ]
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tsZybAeJT5My",
        "colab_type": "text"
      },
      "source": [
        "### Initialization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bGSlM5fmT5M1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# For Colab only!\n",
        "\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cell_style": "split",
        "id": "0XwG0znET5M5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cell_style": "split",
        "id": "BQl-XD9UT5M9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch.nn import functional as F\n",
        "from torchtext import data, datasets\n",
        "from torch import nn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cell_style": "split",
        "scrolled": true,
        "id": "Jo8WpQW9T5NA",
        "colab_type": "code",
        "outputId": "6aa9fd68-74f7-463c-8094-26479b7a5405",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "print(tf.__version__)\n",
        "print(tf.test.is_gpu_available())"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0-rc1\n",
            "WARNING:tensorflow:From <ipython-input-4-2e82a26757ae>:2: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.config.list_physical_devices('GPU')` instead.\n",
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cell_style": "split",
        "id": "zKAqsnciT5NF",
        "colab_type": "code",
        "outputId": "101c8603-549b-483f-db3e-bd09dc1ba88b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(torch.__version__)\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.4.0\n",
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CshDQz-RT5NJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "num_epochs = 10\n",
        "batch_size = 128\n",
        "learning_rate = 0.001\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FCxxG0FxQzEF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "total_words = 10000\n",
        "max_review_words = 200\n",
        "embedding_len = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i-PslWs3T5NI",
        "colab_type": "text"
      },
      "source": [
        "### Tensorflow\n",
        "IMDB data set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g39-0UeWlb4k",
        "colab_type": "text"
      },
      "source": [
        "#### Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q55PDLjpQueb",
        "colab_type": "code",
        "outputId": "16186cd2-49e9-4d89-dcae-ad44da7654a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "(x_train, y_train),(x_test, y_test) = keras.datasets.imdb.load_data(num_words = total_words)\n",
        "print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25000,) (25000,) (25000,) (25000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajO-ZEfk4Ij_",
        "colab_type": "code",
        "outputId": "42aa747a-6f67-4085-aa54-923c422a71b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# x_train = keras.preprocessing.sequence.pad_sequences(x_train)\n",
        "# x_test = keras.preprocessing.sequence.pad_sequences(x_test)\n",
        "\n",
        "# print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25000, 2494) (25000,) (25000, 2315) (25000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4m_AVK7oRCvj",
        "colab_type": "code",
        "outputId": "4d95114c-497c-426a-e843-45ed24deb38d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "## pad sequence to the same length\n",
        "x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen = max_review_words)\n",
        "x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen = max_review_words)\n",
        "\n",
        "print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25000, 200) (25000,) (25000, 200) (25000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8G65seyWIe9h",
        "colab_type": "code",
        "outputId": "b13b894b-b98a-47e3-8a85-ad7be3c38adb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "# See an actual review in words\n",
        "# Reverse from integers to words using the DICTIONARY (given by keras...need to do nothing to create it)\n",
        "\n",
        "word_index = keras.datasets.imdb.get_word_index()\n",
        "\n",
        "reverse_word_index = dict(\n",
        "[(value, key) for (key, value) in word_index.items()])\n",
        "\n",
        "decoded_review = ' '.join(\n",
        "[reverse_word_index.get(i - 3, '?') for i in x_train[1]])\n",
        "\n",
        "print(decoded_review)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json\n",
            "1646592/1641221 [==============================] - 0s 0us/step\n",
            "? ? ? ? ? ? ? ? ? ? ? ? big hair big boobs bad music and a giant safety pin these are the words to best describe this terrible movie i love cheesy horror movies and i've seen hundreds but this had got to be on of the worst ever made the plot is paper thin and ridiculous the acting is an abomination the script is completely laughable the best is the end showdown with the cop and how he worked out who the killer is it's just so damn terribly written the clothes are sickening and funny in equal ? the hair is big lots of boobs ? men wear those cut ? shirts that show off their ? sickening that men actually wore them and the music is just ? trash that plays over and over again in almost every scene there is trashy music boobs and ? taking away bodies and the gym still doesn't close for ? all joking aside this is a truly bad film whose only charm is to look back on the disaster that was the 80's and have a good old laugh at how bad everything was back then\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jYGv6XURWhP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
        "ds_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lK_c1TAaRm88",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds_train=ds_train.shuffle(1000).batch(batch_size, drop_remainder = True)\n",
        "ds_test=ds_test.shuffle(1000).batch(batch_size, drop_remainder = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OiVc3ZtxSPcV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(type(ds_train))\n",
        "data, label = next(iter(ds_train))\n",
        "print(data.shape, label.shape)\n",
        "\n",
        "decoded_review = ' '.join(\n",
        "[reverse_word_index.get(i - 3, '?') for i in data[0].numpy()])\n",
        "\n",
        "print(decoded_review)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vDX0rbORT5Nr",
        "colab_type": "text"
      },
      "source": [
        "#### SimpleRNN\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fkmqyFxdU2fZ",
        "colab_type": "text"
      },
      "source": [
        "Tensorflow\n",
        "\n",
        "[b,max_review_words] --> **embedding** --> [b, max_review_words, embedding_len]\n",
        "\n",
        "[b, max_review_words, embedding_len]--> **RNN/LSTM/GRU layers** with *num_units* -->[b,num_units ]\n",
        "\n",
        "[b, num_units] --> [b, 1]\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cell_style": "split",
        "id": "-h_iJsUiT5Nr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RNN_model(keras.Model):\n",
        "    def __init__(self, num_units):\n",
        "        super().__init__()\n",
        "    \n",
        "        # embedding   [b, 80] ->[b, 80 embedding_len=100]  \n",
        "        # self.embedding = layers.Embedding(input_dim=total_words,output_dim=embedding_len,input_length=max_review_words)\n",
        "        self.embedding = layers.Embedding(input_dim=total_words,output_dim=embedding_len)\n",
        "        # [b, 80, 100] ->[b, num_units]\n",
        "        self.RNN1 = layers.SimpleRNN(units=num_units,dropout=0.5, return_sequences=True)\n",
        "        self.RNN2 = layers.SimpleRNN(units=num_units,dropout=0.5, return_sequences=False)\n",
        "        \n",
        "        # [b, num_units] ->[b,1]\n",
        "        self.fc = layers.Dense(1)\n",
        "    \n",
        "    def call(self,x, training = None):\n",
        "        # print(\"input shape: \", x.shape)       \n",
        "        outputs = self.embedding(x)\n",
        "\n",
        "        # print(\"embedding output shape: \", outputs.shape)       \n",
        "        outputs = self.RNN1(outputs, training = training)\n",
        "        outputs = self.RNN2(outputs, training = training)\n",
        "        \n",
        "        # print(\"RNN output shape: \", outputs.shape)\n",
        "        outputs = self.fc(outputs)\n",
        "        # print(\"fc output shape\", outputs.shape)\n",
        "        return outputs\n",
        "    \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LRCRDiskLvY5",
        "colab_type": "text"
      },
      "source": [
        "#### GRU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hThW4CyNc3ms",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class GRU_model(keras.Model):\n",
        "    def __init__(self, num_units):\n",
        "        super().__init__()\n",
        "    \n",
        "        # embedding   [b, 80] ->[b, 80 embedding_len=100]  \n",
        "        self.embedding = layers.Embedding(input_dim=total_words,output_dim=embedding_len,input_length=max_review_words)\n",
        "        \n",
        "        # [b, 80, 100] ->[b, num_units]\n",
        "        self.GRU1 = layers.GRU(units=num_units,dropout=0.5, return_sequences=True)\n",
        "        self.GRU2 = layers.GRU(units=num_units,dropout=0.5, return_sequences=False)\n",
        "        \n",
        "        # [b, num_units] ->[b,1] \n",
        "        self.fc = layers.Dense(1)\n",
        "\n",
        "\n",
        "    \n",
        "    def call(self,x, training = None):\n",
        "        # print(\"input shape: \", x.shape)\n",
        "        \n",
        "        outputs = self.embedding(x)\n",
        "\n",
        "        # print(\"embedding output shape: \", outputs.shape)\n",
        "        \n",
        "        outputs = self.GRU1(outputs, training = training)\n",
        "        outputs = self.GRU2(outputs, training = training)\n",
        "        \n",
        "        # print(\"RNN output shape: \", outputs.shape)\n",
        "\n",
        "        outputs = self.fc(outputs)\n",
        "\n",
        "        # print(\"fc output shape\", outputs.shape)\n",
        "        return outputs\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pUaw78k6LfNq",
        "colab_type": "text"
      },
      "source": [
        "#### LSTM\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3eVCFIUOmQv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTM_model(keras.Model):\n",
        "    def __init__(self, num_units):\n",
        "        super().__init__()\n",
        "    \n",
        "        # embedding   [b, 80] ->[b, 80 embedding_len=100]  \n",
        "        self.embedding = layers.Embedding(input_dim=total_words,output_dim=embedding_len,input_length=max_review_words)\n",
        "        \n",
        "        # [b, 80, 100] ->[b, num_units]\n",
        "        self.LSTM1 = layers.LSTM(units=num_units,dropout=0.5, return_sequences=True)\n",
        "        self.LSTM2 = layers.LSTM(units=num_units,dropout=0.5, return_sequences=False)\n",
        "        \n",
        "        # [b, num_units] ->[b,1] \n",
        "        self.fc = layers.Dense(1)\n",
        "\n",
        "\n",
        "    \n",
        "    def call(self,x, training = None):\n",
        "        # print(\"input shape: \", x.shape)\n",
        "        \n",
        "        outputs = self.embedding(x)\n",
        "\n",
        "        # print(\"embedding output shape: \", outputs.shape)\n",
        "        \n",
        "        outputs = self.LSTM1(outputs, training = training)\n",
        "        outputs = self.LSTM2(outputs, training = training)\n",
        "        \n",
        "        # print(\"RNN output shape: \", outputs.shape)\n",
        "\n",
        "        outputs = self.fc(outputs)\n",
        "\n",
        "        # print(\"fc output shape\", outputs.shape)\n",
        "        return outputs\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E7_h-E05EhUI",
        "colab_type": "text"
      },
      "source": [
        "#### Quick fit"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_mpgaGwaC9i",
        "colab_type": "code",
        "outputId": "c0299d40-c910-464c-f609-35a58b260f65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        }
      },
      "source": [
        "model = RNN_model(64)\n",
        "model.build(input_shape=(None, max_review_words))\n",
        "model.summary()\n",
        "\n",
        "model.compile(optimizer=keras.optimizers.Adam(learning_rate),\n",
        "              loss = tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics = ['accuracy'])\n",
        "history = model.fit(ds_train, epochs=num_epochs, validation_data=ds_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"rnn_model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      multiple                  1000000   \n",
            "_________________________________________________________________\n",
            "simple_rnn_2 (SimpleRNN)     multiple                  10560     \n",
            "_________________________________________________________________\n",
            "simple_rnn_3 (SimpleRNN)     multiple                  8256      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              multiple                  65        \n",
            "=================================================================\n",
            "Total params: 1,018,881\n",
            "Trainable params: 1,018,881\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Train for 195 steps, validate for 195 steps\n",
            "Epoch 1/10\n",
            "195/195 [==============================] - 1083s 6s/step - loss: 0.6836 - accuracy: 0.5353 - val_loss: 0.6302 - val_accuracy: 0.6462\n",
            "Epoch 2/10\n",
            "195/195 [==============================] - 1077s 6s/step - loss: 0.5262 - accuracy: 0.7260 - val_loss: 0.5033 - val_accuracy: 0.7886\n",
            "Epoch 3/10\n",
            "195/195 [==============================] - 1056s 5s/step - loss: 0.4478 - accuracy: 0.7916 - val_loss: 0.4383 - val_accuracy: 0.8058\n",
            "Epoch 4/10\n",
            " 74/195 [==========>...................] - ETA: 10:09 - loss: 0.4145 - accuracy: 0.8147"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfh_12tBIQvN",
        "colab_type": "code",
        "outputId": "517cd938-7dc6-40fe-904f-d215bc9e2546",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "model = GRU_model(64)\n",
        "model.build(input_shape=(None, max_review_words))\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"gru_model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      multiple                  1000000   \n",
            "_________________________________________________________________\n",
            "gru (GRU)                    multiple                  31872     \n",
            "_________________________________________________________________\n",
            "gru_1 (GRU)                  multiple                  24960     \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              multiple                  65        \n",
            "=================================================================\n",
            "Total params: 1,056,897\n",
            "Trainable params: 1,056,897\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKxhnVjfL4I_",
        "colab_type": "text"
      },
      "source": [
        "#### TF2.0 Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RewTSE2iHoyG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = keras.optimizers.Adam(learning_rate= learning_rate)\n",
        "\n",
        "categorical_accuracy_train = keras.metrics.BinaryAccuracy()\n",
        "categorical_accuracy_test = keras.metrics.BinaryAccuracy()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GfOJ6rowHepB",
        "colab_type": "code",
        "outputId": "35dc8f63-13a8-45d0-a8e7-fabff4e4c68a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "for epoch in range(num_epochs):\n",
        "  for step, (x, y) in enumerate(ds_train):\n",
        "    y = tf.reshape(y,[-1,1])\n",
        "    with tf.GradientTape() as tape:\n",
        "      logits = model(x, training = True)\n",
        "      loss = tf.keras.losses.binary_crossentropy(y_true=y, y_pred=logits, from_logits=True)\n",
        "      loss = tf.reduce_mean(loss)  \n",
        "    \n",
        "    grad = tape.gradient(loss, model.trainable_variables)\n",
        "    optimizer.apply_gradients(grads_and_vars= zip(grad, model.trainable_variables))\n",
        "\n",
        "    [x_train, y_train] = next(iter(ds_train))\n",
        "    train_logits = model(x_train, training = False)\n",
        "    categorical_accuracy_train.update_state(y_true = y_train, y_pred=train_logits)\n",
        "    train_accuracy = categorical_accuracy_train.result().numpy()\n",
        "    \n",
        "    [x_test, y_test] = next(iter(ds_test))\n",
        "    logits = model(x_test, training = False)\n",
        "    categorical_accuracy_test.update_state(y_true = y_test, y_pred=logits)\n",
        "    accuracy = categorical_accuracy_test.result().numpy()\n",
        "\n",
        "    if step%20 == 0:\n",
        "      print(\"epoch: {}, step: {}, loss: {}, train_accuracy: {} test_accuracy: {}\".format(epoch, step, loss.numpy(),train_accuracy,accuracy))\n",
        "\n",
        "      # with summary_writer.as_default():\n",
        "      #   tf.summary.scalar(\"loss epoch: \"+str(epoch), loss.numpy(), step = step)\n",
        "      #   tf.summary.scalar(\"test_acc epoch: \"+str(epoch), accuracy, step = step)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch: 0, step: 0, loss: 0.6929845213890076, train_accuracy: 0.4921875 test_accuracy: 0.5\n",
            "epoch: 0, step: 20, loss: 0.7019393444061279, train_accuracy: 0.4832589328289032 test_accuracy: 0.5372023582458496\n",
            "epoch: 0, step: 40, loss: 0.6733893156051636, train_accuracy: 0.49333080649375916 test_accuracy: 0.535442054271698\n",
            "epoch: 0, step: 60, loss: 0.6110574007034302, train_accuracy: 0.5466188788414001 test_accuracy: 0.5751792788505554\n",
            "epoch: 0, step: 80, loss: 0.4642757177352905, train_accuracy: 0.6062885522842407 test_accuracy: 0.6302083134651184\n",
            "epoch: 0, step: 100, loss: 0.4546666145324707, train_accuracy: 0.6482828259468079 test_accuracy: 0.664371907711029\n",
            "epoch: 0, step: 120, loss: 0.4467814862728119, train_accuracy: 0.6799457669258118 test_accuracy: 0.6900180578231812\n",
            "epoch: 0, step: 140, loss: 0.3914114236831665, train_accuracy: 0.7045655846595764 test_accuracy: 0.7134308218955994\n",
            "epoch: 0, step: 160, loss: 0.37333858013153076, train_accuracy: 0.7238936424255371 test_accuracy: 0.7298622131347656\n",
            "epoch: 0, step: 180, loss: 0.3636167645454407, train_accuracy: 0.7388639450073242 test_accuracy: 0.7446478009223938\n",
            "epoch: 1, step: 0, loss: 0.3098519444465637, train_accuracy: 0.7501993179321289 test_accuracy: 0.754902720451355\n",
            "epoch: 1, step: 20, loss: 0.23841191828250885, train_accuracy: 0.7641782164573669 test_accuracy: 0.7652271389961243\n",
            "epoch: 1, step: 40, loss: 0.288510262966156, train_accuracy: 0.7776747941970825 test_accuracy: 0.7728085517883301\n",
            "epoch: 1, step: 60, loss: 0.16597682237625122, train_accuracy: 0.7899169921875 test_accuracy: 0.781524658203125\n",
            "epoch: 1, step: 80, loss: 0.2571181058883667, train_accuracy: 0.8002434372901917 test_accuracy: 0.7881850004196167\n",
            "epoch: 1, step: 100, loss: 0.2639066278934479, train_accuracy: 0.8084881901741028 test_accuracy: 0.7933118939399719\n",
            "epoch: 1, step: 120, loss: 0.21772146224975586, train_accuracy: 0.815664529800415 test_accuracy: 0.7979133725166321\n",
            "epoch: 1, step: 140, loss: 0.2143445461988449, train_accuracy: 0.8216145634651184 test_accuracy: 0.8021065592765808\n",
            "epoch: 1, step: 160, loss: 0.3343116044998169, train_accuracy: 0.8264132738113403 test_accuracy: 0.8050386309623718\n",
            "epoch: 1, step: 180, loss: 0.23301611840724945, train_accuracy: 0.831200122833252 test_accuracy: 0.8083651661872864\n",
            "epoch: 2, step: 0, loss: 0.21133843064308167, train_accuracy: 0.8346187472343445 test_accuracy: 0.8107816576957703\n",
            "epoch: 2, step: 20, loss: 0.2984205484390259, train_accuracy: 0.8400813341140747 test_accuracy: 0.813431441783905\n",
            "epoch: 2, step: 40, loss: 0.20200824737548828, train_accuracy: 0.8453813791275024 test_accuracy: 0.8151645660400391\n",
            "epoch: 2, step: 60, loss: 0.1921882927417755, train_accuracy: 0.8504538536071777 test_accuracy: 0.8169345855712891\n",
            "epoch: 2, step: 80, loss: 0.19168983399868011, train_accuracy: 0.8549462556838989 test_accuracy: 0.8194665312767029\n",
            "epoch: 2, step: 100, loss: 0.1861107051372528, train_accuracy: 0.858929455280304 test_accuracy: 0.8206466436386108\n",
            "epoch: 2, step: 120, loss: 0.18360257148742676, train_accuracy: 0.8625244498252869 test_accuracy: 0.8222236037254333\n",
            "epoch: 2, step: 140, loss: 0.2356555312871933, train_accuracy: 0.8653631210327148 test_accuracy: 0.8240495324134827\n",
            "epoch: 2, step: 160, loss: 0.18713995814323425, train_accuracy: 0.8682650923728943 test_accuracy: 0.8253034353256226\n",
            "epoch: 2, step: 180, loss: 0.19510455429553986, train_accuracy: 0.8710868954658508 test_accuracy: 0.8266199827194214\n",
            "epoch: 3, step: 0, loss: 0.09733401238918304, train_accuracy: 0.8732801675796509 test_accuracy: 0.8276983499526978\n",
            "epoch: 3, step: 20, loss: 0.18549758195877075, train_accuracy: 0.8763536214828491 test_accuracy: 0.8286922574043274\n",
            "epoch: 3, step: 40, loss: 0.2044672816991806, train_accuracy: 0.8795177936553955 test_accuracy: 0.8290610313415527\n",
            "epoch: 3, step: 60, loss: 0.17291828989982605, train_accuracy: 0.8821715116500854 test_accuracy: 0.8300358057022095\n",
            "epoch: 3, step: 80, loss: 0.1910155713558197, train_accuracy: 0.8847480416297913 test_accuracy: 0.8306470513343811\n",
            "epoch: 3, step: 100, loss: 0.15340012311935425, train_accuracy: 0.8870945572853088 test_accuracy: 0.8313365578651428\n",
            "epoch: 3, step: 120, loss: 0.20469513535499573, train_accuracy: 0.889263927936554 test_accuracy: 0.8324849605560303\n",
            "epoch: 3, step: 140, loss: 0.22637557983398438, train_accuracy: 0.8910661935806274 test_accuracy: 0.8334086537361145\n",
            "epoch: 3, step: 160, loss: 0.11059622466564178, train_accuracy: 0.8928975462913513 test_accuracy: 0.834335207939148\n",
            "epoch: 3, step: 180, loss: 0.1851995438337326, train_accuracy: 0.8944292664527893 test_accuracy: 0.8351419568061829\n",
            "epoch: 4, step: 0, loss: 0.07493126392364502, train_accuracy: 0.8957566618919373 test_accuracy: 0.8356574177742004\n",
            "epoch: 4, step: 20, loss: 0.1631879210472107, train_accuracy: 0.8977059721946716 test_accuracy: 0.8360350131988525\n",
            "epoch: 4, step: 40, loss: 0.20116321742534637, train_accuracy: 0.8997887372970581 test_accuracy: 0.8364989161491394\n",
            "epoch: 4, step: 60, loss: 0.13769970834255219, train_accuracy: 0.9018281698226929 test_accuracy: 0.8370708227157593\n",
            "epoch: 4, step: 80, loss: 0.10902083665132523, train_accuracy: 0.9037002921104431 test_accuracy: 0.8377885222434998\n",
            "epoch: 4, step: 100, loss: 0.2441059648990631, train_accuracy: 0.9054341912269592 test_accuracy: 0.8382342457771301\n",
            "epoch: 4, step: 120, loss: 0.13754889369010925, train_accuracy: 0.9071431159973145 test_accuracy: 0.8385908007621765\n",
            "epoch: 4, step: 140, loss: 0.15475720167160034, train_accuracy: 0.9085148572921753 test_accuracy: 0.8390336632728577\n",
            "epoch: 4, step: 160, loss: 0.1307254433631897, train_accuracy: 0.9098449349403381 test_accuracy: 0.8393829464912415\n",
            "epoch: 4, step: 180, loss: 0.08809919655323029, train_accuracy: 0.9110057950019836 test_accuracy: 0.8397177457809448\n",
            "epoch: 5, step: 0, loss: 0.13164187967777252, train_accuracy: 0.9119812846183777 test_accuracy: 0.8401479125022888\n",
            "epoch: 5, step: 20, loss: 0.1828668713569641, train_accuracy: 0.9134349822998047 test_accuracy: 0.8403300642967224\n",
            "epoch: 5, step: 40, loss: 0.12038646638393402, train_accuracy: 0.9149160385131836 test_accuracy: 0.8403897285461426\n",
            "epoch: 5, step: 60, loss: 0.13099664449691772, train_accuracy: 0.9164228439331055 test_accuracy: 0.8409598469734192\n",
            "epoch: 5, step: 80, loss: 0.11982254683971405, train_accuracy: 0.9177393913269043 test_accuracy: 0.8413233757019043\n",
            "epoch: 5, step: 100, loss: 0.10797316581010818, train_accuracy: 0.9190505743026733 test_accuracy: 0.8415499925613403\n",
            "epoch: 5, step: 120, loss: 0.10130858421325684, train_accuracy: 0.9203139543533325 test_accuracy: 0.8418182730674744\n",
            "epoch: 5, step: 140, loss: 0.1610574722290039, train_accuracy: 0.9215319752693176 test_accuracy: 0.8419438600540161\n",
            "epoch: 5, step: 160, loss: 0.038708511739969254, train_accuracy: 0.9225008487701416 test_accuracy: 0.8421682715415955\n",
            "epoch: 5, step: 180, loss: 0.08266238868236542, train_accuracy: 0.9234766960144043 test_accuracy: 0.8423240184783936\n",
            "epoch: 6, step: 0, loss: 0.10446806252002716, train_accuracy: 0.9242568016052246 test_accuracy: 0.8424823880195618\n",
            "epoch: 6, step: 20, loss: 0.06318919360637665, train_accuracy: 0.9253909587860107 test_accuracy: 0.8428972363471985\n",
            "epoch: 6, step: 40, loss: 0.1336776614189148, train_accuracy: 0.9264102578163147 test_accuracy: 0.8429113626480103\n",
            "epoch: 6, step: 60, loss: 0.053849272429943085, train_accuracy: 0.9274979829788208 test_accuracy: 0.8432486057281494\n",
            "epoch: 6, step: 80, loss: 0.06864888221025467, train_accuracy: 0.9285571575164795 test_accuracy: 0.8431941866874695\n",
            "epoch: 6, step: 100, loss: 0.08279352635145187, train_accuracy: 0.9295768737792969 test_accuracy: 0.8430984616279602\n",
            "epoch: 6, step: 120, loss: 0.04418952763080597, train_accuracy: 0.9303894639015198 test_accuracy: 0.8431690335273743\n",
            "epoch: 6, step: 140, loss: 0.17433500289916992, train_accuracy: 0.9310342669487 test_accuracy: 0.8433209657669067\n",
            "epoch: 6, step: 160, loss: 0.09702035784721375, train_accuracy: 0.9318064451217651 test_accuracy: 0.843703031539917\n",
            "epoch: 6, step: 180, loss: 0.03447684645652771, train_accuracy: 0.9325383901596069 test_accuracy: 0.8439119458198547\n",
            "epoch: 7, step: 0, loss: 0.047563016414642334, train_accuracy: 0.9330618381500244 test_accuracy: 0.8440588116645813\n",
            "epoch: 7, step: 20, loss: 0.07108097523450851, train_accuracy: 0.9338868856430054 test_accuracy: 0.8441389203071594\n",
            "epoch: 7, step: 40, loss: 0.04804609343409538, train_accuracy: 0.9347328543663025 test_accuracy: 0.8443945646286011\n",
            "epoch: 7, step: 60, loss: 0.055411793291568756, train_accuracy: 0.935538649559021 test_accuracy: 0.8447635173797607\n",
            "epoch: 7, step: 80, loss: 0.13299480080604553, train_accuracy: 0.9362843632698059 test_accuracy: 0.8449602127075195\n",
            "epoch: 7, step: 100, loss: 0.052347149699926376, train_accuracy: 0.9370523691177368 test_accuracy: 0.8451995253562927\n",
            "epoch: 7, step: 120, loss: 0.03549407422542572, train_accuracy: 0.9377734065055847 test_accuracy: 0.8454376459121704\n",
            "epoch: 7, step: 140, loss: 0.03824343904852867, train_accuracy: 0.9384545087814331 test_accuracy: 0.8457057476043701\n",
            "epoch: 7, step: 160, loss: 0.058421239256858826, train_accuracy: 0.939112663269043 test_accuracy: 0.8458848595619202\n",
            "epoch: 7, step: 180, loss: 0.05873472988605499, train_accuracy: 0.939758837223053 test_accuracy: 0.8460695147514343\n",
            "epoch: 8, step: 0, loss: 0.0758453905582428, train_accuracy: 0.940212607383728 test_accuracy: 0.8462473750114441\n",
            "epoch: 8, step: 20, loss: 0.08883759379386902, train_accuracy: 0.9409096240997314 test_accuracy: 0.8465073704719543\n",
            "epoch: 8, step: 40, loss: 0.04594291001558304, train_accuracy: 0.9416185021400452 test_accuracy: 0.8466387987136841\n",
            "epoch: 8, step: 60, loss: 0.05582893267273903, train_accuracy: 0.9422954320907593 test_accuracy: 0.8468537926673889\n",
            "epoch: 8, step: 80, loss: 0.09056856483221054, train_accuracy: 0.9429511427879333 test_accuracy: 0.8470444679260254\n",
            "epoch: 8, step: 100, loss: 0.11353901028633118, train_accuracy: 0.9435533881187439 test_accuracy: 0.8473058342933655\n",
            "epoch: 8, step: 120, loss: 0.08912180364131927, train_accuracy: 0.9441227316856384 test_accuracy: 0.8473983407020569\n",
            "epoch: 8, step: 140, loss: 0.022785233333706856, train_accuracy: 0.9446097612380981 test_accuracy: 0.8475207686424255\n",
            "epoch: 8, step: 160, loss: 0.07538655400276184, train_accuracy: 0.9450492262840271 test_accuracy: 0.8476267457008362\n",
            "epoch: 8, step: 180, loss: 0.12652812898159027, train_accuracy: 0.9455368518829346 test_accuracy: 0.8477078676223755\n",
            "epoch: 9, step: 0, loss: 0.0540345199406147, train_accuracy: 0.9458953142166138 test_accuracy: 0.847749650478363\n",
            "epoch: 9, step: 20, loss: 0.0387505367398262, train_accuracy: 0.9464254379272461 test_accuracy: 0.8478630185127258\n",
            "epoch: 9, step: 40, loss: 0.039219796657562256, train_accuracy: 0.94700026512146 test_accuracy: 0.8481304049491882\n",
            "epoch: 9, step: 60, loss: 0.03598577901721001, train_accuracy: 0.9475538730621338 test_accuracy: 0.8482370376586914\n",
            "epoch: 9, step: 80, loss: 0.03215416520833969, train_accuracy: 0.9480826258659363 test_accuracy: 0.8483753800392151\n",
            "epoch: 9, step: 100, loss: 0.01966116763651371, train_accuracy: 0.9485494494438171 test_accuracy: 0.8483507633209229\n",
            "epoch: 9, step: 120, loss: 0.03980546444654465, train_accuracy: 0.9490313529968262 test_accuracy: 0.8484932780265808\n",
            "epoch: 9, step: 140, loss: 0.04519718885421753, train_accuracy: 0.9494866132736206 test_accuracy: 0.8485215306282043\n",
            "epoch: 9, step: 160, loss: 0.02868777886033058, train_accuracy: 0.9499363899230957 test_accuracy: 0.8486185669898987\n",
            "epoch: 9, step: 180, loss: 0.029568195343017578, train_accuracy: 0.9503163695335388 test_accuracy: 0.8486973643302917\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zI7zP4RHZRpM",
        "colab_type": "text"
      },
      "source": [
        "### PyTorch\n",
        "\n",
        "Tensorflow\n",
        "\n",
        "[seq_len, b] --> **embedding** --> [seq_len, b, embedding_len]\n",
        "\n",
        "[seq_len, b, embedding_len]--> **RNN/LSTM/GRU layers** with *hidden_dim* -->[seq_len, b, hidden_dim]\n",
        "---> only use the last seq [b, hidden_dim]\n",
        "\n",
        "[b, hidden_dim] --> [b, 1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nH2-ul_3N7ec",
        "colab_type": "text"
      },
      "source": [
        "#### Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PXNcwrXYN9yC",
        "colab_type": "code",
        "outputId": "5e7d2943-388b-466e-b7f3-5446aad20fc6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "TEXT = data.Field(tokenize='spacy')\n",
        "LABEL = data.LabelField(dtype=torch.float)\n",
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading aclImdb_v1.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:07<00:00, 11.0MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v5SxijgsRET1",
        "colab_type": "code",
        "outputId": "2b2bb2fa-8b59-409a-e59f-8b1bd5755f57",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print('len of train data:', len(train_data))\n",
        "print('len of test data:', len(test_data))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "len of train data: 25000\n",
            "len of test data: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoEIBkE6Rgrj",
        "colab_type": "code",
        "outputId": "49860773-5f87-4766-b7a4-65a93e1fe702",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "print(train_data.examples[232].text)\n",
        "print(train_data.examples[232].label)\n",
        "print(len(train_data.examples[232].text))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Having', 'borrowed', 'this', 'movie', 'from', 'the', 'local', 'library', 'a', 'couple', 'of', 'weeks', 'ago', 'intending', 'to', 'originally', 'see', 'this', 'on', 'or', 'a', 'few', 'days', 'after', 'Memorial', 'Day', ',', 'I', 'finally', 'got', 'to', 'seeing', 'Sayonara', 'just', 'this', 'morning', '.', 'In', 'this', 'one', 'Marlon', 'Brando', 'plays', 'Major', 'Lloyd', '\"', 'Ace', '\"', 'Gruver', ',', 'a', 'General', \"'s\", 'son', 'who', \"'s\", 'been', 'raised', 'a', 'certain', 'way', ',', 'being', 'transfered', 'from', 'Korea', 'to', 'Japan', 'where', 'his', 'girlfriend', 'Eileen', 'Webster', '(', 'Patricia', 'Owens', ')', 'conveniently', 'happens', 'to', 'be', '.', 'Before', 'leaving', ',', 'he', 'tries', 'to', 'persuade', 'one', 'of', 'his', 'men', ',', 'a', 'Joe', 'Kelly', '(', 'Red', 'Buttons', ')', ',', 'out', 'of', 'marrying', 'Japanese', 'woman', 'Katsumi', '(', 'Miyoshi', 'Umeki', ')', 'since', 'that', \"'s\", 'a', 'violation', 'of', 'military', 'fraternization', 'laws', '.', 'With', 'the', 'romance', 'of', 'him', 'and', 'Eileen', 'on', 'the', 'outs', ',', 'however', ',', 'Ace', 'not', 'only', 'becomes', 'the', 'best', 'man', 'at', 'Joe', 'and', 'Katsumi', \"'s\", 'wedding', ',', 'he', 'falls', 'for', 'an', 'Asian', 'himself', 'after', 'he', 'and', 'Captain', 'Mike', 'Bailey', '(', 'James', 'Garner', ')', 'go', 'out', 'on', 'the', 'town', 'and', 'watch', 'headlining', 'entertainer', 'Hana', '-', 'ogi', '(', 'Miiko', 'Taka', ')', 'on', 'stage', '.', 'Bailey', 'himself', 'is', 'dating', 'one', 'of', 'the', 'dancers', ',', 'Fumiko', '-', 'San', '(', 'Reiko', 'Kuba', ')', '.', 'Eileen', 'herself', 'seems', 'to', 'have', 'a', 'fancy', 'for', 'one', 'of', 'the', 'Kabuki', 'performers', ',', 'Nakamura', '(', 'Ricardo', 'Montalban', ')', '.', 'I', \"'ll\", 'stop', 'right', 'there', 'and', 'say', 'that', 'this', 'was', 'a', 'mostly', 'compelling', 'drama', 'about', 'the', 'prejudices', 'concerning', 'American', '-', 'Asian', 'relations', 'of', 'romance', 'that', 'was', 'very', 'touching', 'from', 'beginning', 'to', 'end', '.', 'Even', 'seeing', 'Hispanic', 'Montalban', 'playing', 'an', 'Oriental', 'is', \"n't\", 'too', 'embarrassing', '(', 'though', 'it', \"'s\", 'a', 'good', 'thing', 'his', 'part', 'is', 'short', ')', '.', 'And', 'there', \"'s\", 'some', 'nice', 'touches', 'of', 'humor', 'like', 'that', 'of', 'Brando', \"'s\", 'head', 'hitting', 'the', 'top', 'of', 'Button', \"'s\", 'and', 'Umeki', \"'s\", 'inside', 'doorway', 'more', 'than', 'once', '.', 'Red', 'and', 'Miyoshi', 'themselves', 'deserve', 'their', 'Oscars', 'especially', 'Red', 'with', 'his', 'defiant', 'and', 'proud', 'emotions', 'throughout', '.', 'Rookie', 'Garner', ',', 'before', 'being', 'cast', 'in', 'his', 'legendary', 'role', 'on', 'TV', \"'s\", '\"', 'Marverick', '\"', ',', 'is', 'fine', 'in', 'his', 'scenes', 'with', 'Brando', 'and', 'Miiko', 'Taka', 'shows', 'great', 'restraint', 'in', 'her', 'initial', 'characterization', 'as', 'an', 'anti', '-', 'American', '.', 'While', 'I', \"'ve\", 'read', 'there', 'were', 'some', 'changes', 'from', 'James', 'Michener', \"'s\", 'novel', ',', 'I', 'ca', \"n't\", 'imagine', 'director', 'Josha', 'Logan', ',', 'who', 'had', 'previously', 'adopted', 'another', 'Michener', 'work', 'into', 'the', 'Broadway', 'musical', '\"', 'South', 'Pacific', '\"', 'and', 'would', 'eventually', 'make', 'that', 'into', 'a', 'movie', 'as', 'well', ',', 'not', 'staying', 'true', 'to', 'the', 'original', 'source', '.', 'He', 'certainly', 'provided', 'some', 'inspiration', 'with', 'the', 'ending', 'scenes', 'that', 'made', 'the', 'heartbreaking', 'earlier', 'tragedy', 'in', 'the', 'film', 'a', 'somewhat', 'necessary', 'plot', 'twist', '.', 'Some', 'of', 'the', 'production', 'numbers', 'may', 'have', 'made', 'the', 'movie', 'a', 'little', 'longish', 'but', 'otherwise', ',', 'Sayonara', 'was', 'wonderful', 'educational', 'experience', 'about', 'the', \"'\", '50s', 'mores', 'that', 'permeated', 'America', 'and', 'Japan', 'at', 'the', 'time', '.']\n",
            "pos\n",
            "479\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTTTpAWFRrHU",
        "colab_type": "code",
        "outputId": "dd5bb715-408b-46b3-cb68-c1805e9bbfc2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# word2vec, glove\n",
        "TEXT.build_vocab(train_data, max_size=10000, vectors='glove.6B.100d')\n",
        "LABEL.build_vocab(train_data)\n",
        "\n",
        "\n",
        "batchsz = 64\n",
        "device = torch.device('cuda')\n",
        "train_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, test_data),\n",
        "    batch_size = batchsz,\n",
        "    device=device\n",
        ")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".vector_cache/glove.6B.zip: 862MB [06:31, 2.20MB/s]                           \n",
            "100%|█████████▉| 398208/400000 [00:15<00:00, 23829.91it/s]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGAZYIBtUA0i",
        "colab_type": "code",
        "outputId": "1b40b056-50d0-4d23-cc91-8d33c8a47a8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "print(type(train_iterator))\n",
        "ds = next(iter(train_iterator))\n",
        "print(ds.text.shape)\n",
        "print(ds.label.shape)\n",
        "# print(ds.text[:,1])\n",
        "# print(ds.label[1])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'torchtext.data.iterator.BucketIterator'>\n",
            "torch.Size([940, 64])\n",
            "torch.Size([64])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K36POWFuWryQ",
        "colab_type": "text"
      },
      "source": [
        "#### RNN module"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wam9XfOWqmM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RNN_nn(nn.Module):\n",
        "\n",
        "  def __init__(self, vocab_size, hidden_dim):\n",
        "    super().__init__()\n",
        "\n",
        "    self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_len)\n",
        "    self.rnn = nn.RNN(input_size=embedding_len, hidden_size=hidden_dim, num_layers=2, dropout=0.5)\n",
        "    self.fc = nn.Linear(in_features=hidden_dim, out_features=1)\n",
        "\n",
        "  def forward(self, x):  \n",
        "\n",
        "    \n",
        "    #[seq_len, b] -> [seq_len, b, embedding_len = 100]\n",
        "    output = self.embedding(x)\n",
        "    # print('embedding size: ', output.shape)\n",
        "    \n",
        "    #[seq_len, b, embedding_len] ->\n",
        "    # out: [seq_len, b, hidden_dim]\n",
        "    # h:   [n_layer=2, b, hidden_dim]\n",
        "    # last seq of out == last layer of h\n",
        "    output, h = self.rnn(output)\n",
        "\n",
        "\n",
        "    # Last seq of the output\n",
        "    out = output[-1:,:,:].squeeze()\n",
        "    \n",
        "    out = self.fc(out)\n",
        "\n",
        "    return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_-hBdDK1Prh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class GRU_nn(nn.Module):\n",
        "\n",
        "  def __init__(self, vocab_size, hidden_dim):\n",
        "    super().__init__()\n",
        "\n",
        "    self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_len)\n",
        "    self.rnn = nn.GRU(input_size=embedding_len, hidden_size=hidden_dim, num_layers=2, dropout=0.5)\n",
        "    self.fc = nn.Linear(in_features=hidden_dim, out_features=1)\n",
        "\n",
        "  def forward(self, x):  \n",
        "\n",
        "    print('input size: ', x.shape)\n",
        "    #[seq_len, b] -> [seq_len, b, embedding_len = 100]\n",
        "    output = self.embedding(x)\n",
        "    print('embedding size: ', output.shape)\n",
        "    \n",
        "    #[seq_len, b, embedding_len] ->\n",
        "    # out: [seq_len, b, hidden_dim]\n",
        "    # h:   [n_layer=2, b, hidden_dim]\n",
        "    # last seq of out == last layer of h\n",
        "    output, h = self.rnn(output)\n",
        "\n",
        "\n",
        "    # Last seq of the output\n",
        "    out = output[-1:,:,:].squeeze()\n",
        "    \n",
        "    out = self.fc(out)\n",
        "\n",
        "    return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phU9PkSd2_IL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTM_nn(nn.Module):\n",
        "    \n",
        "    def __init__(self, vocab_size, hidden_dim):\n",
        "        \"\"\"\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        \n",
        "        # [0-10001] => [100]\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_len)\n",
        "        # [100] => [256]\n",
        "        self.rnn = nn.LSTM(embedding_len, hidden_dim, num_layers=2, \n",
        "                           bidirectional=True, dropout=0.5)\n",
        "        # [256*2] => [1]\n",
        "        self.fc = nn.Linear(hidden_dim*2, 1)\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        \n",
        "        \n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: [seq_len, b] vs [b, 3, 28, 28]\n",
        "        \"\"\"\n",
        "        # [seq, b, 1] => [seq, b, 100]\n",
        "        embedding = self.dropout(self.embedding(x))\n",
        "        \n",
        "        # output: [seq, b, hid_dim*2]\n",
        "        # hidden/h: [num_layers*2, b, hid_dim]\n",
        "        # cell/c: [num_layers*2, b, hid_di]\n",
        "        output, (hidden, cell) = self.rnn(embedding)\n",
        "        \n",
        "        # [num_layers*2, b, hid_dim] => 2 of [b, hid_dim] => [b, hid_dim*2]\n",
        "        hidden = torch.cat([hidden[-2], hidden[-1]], dim=1)\n",
        "        \n",
        "        # [b, hid_dim*2] => [b, 1]\n",
        "        hidden = self.dropout(hidden)\n",
        "        out = self.fc(hidden)\n",
        "        \n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NtxvEuzZaU85",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test\n",
        "\n",
        "x = torch.randint(1, 10000, (1002,128))\n",
        "\n",
        "model = GRU_nn(total_words,64)\n",
        "\n",
        "out = model(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lbPoSqcWyKYR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def binary_acc(preds, y):\n",
        "    \"\"\"\n",
        "    get accuracy\n",
        "    \"\"\"\n",
        "    preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = torch.eq(preds, y).float()\n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNfZh08UgSNx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "network = LSTM_nn(len(TEXT.vocab), 128).to(device)\n",
        "\n",
        "# pretrained_embedding = TEXT.vocab.vectors\n",
        "# print('pretrained_embedding:', pretrained_embedding.shape)\n",
        "# network.embedding.weight.data.copy_(pretrained_embedding)\n",
        "# print('embedding layer inited.')\n",
        "\n",
        "\n",
        "optimizer = torch.optim.Adam(network.parameters(), lr=learning_rate)\n",
        "criteon = nn.BCEWithLogitsLoss().to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ezyrPGm7corM",
        "colab_type": "code",
        "outputId": "3eca8a15-14f5-46e5-9898-0b388f40916f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        }
      },
      "source": [
        "for epoch in range(10):\n",
        "  for step, batch in enumerate(train_iterator):\n",
        "\n",
        "    pred = network(batch.text).squeeze()\n",
        "\n",
        "    loss = criteon(pred, batch.label)\n",
        "    acc = binary_acc(pred, batch.label).item()\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    if step%100 == 0:\n",
        "      print(\"epoch: {}, step: {}, loss: {}, acc:{}\".format(epoch,step,loss.item(), acc))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch: 0, step: 0, loss: 0.6977689862251282, acc:0.5\n",
            "epoch: 0, step: 100, loss: 0.6727264523506165, acc:0.578125\n",
            "epoch: 0, step: 200, loss: 0.6923737525939941, acc:0.5\n",
            "epoch: 0, step: 300, loss: 0.6616694331169128, acc:0.578125\n",
            "epoch: 1, step: 0, loss: 0.7228837609291077, acc:0.59375\n",
            "epoch: 1, step: 100, loss: 0.5663291811943054, acc:0.75\n",
            "epoch: 1, step: 200, loss: 0.6952507495880127, acc:0.5\n",
            "epoch: 1, step: 300, loss: 0.6980805993080139, acc:0.484375\n",
            "epoch: 2, step: 0, loss: 0.6476609110832214, acc:0.640625\n",
            "epoch: 2, step: 100, loss: 0.6129595041275024, acc:0.671875\n",
            "epoch: 2, step: 200, loss: 0.6854571104049683, acc:0.5625\n",
            "epoch: 2, step: 300, loss: 0.6750308275222778, acc:0.609375\n",
            "epoch: 3, step: 0, loss: 0.6261641383171082, acc:0.671875\n",
            "epoch: 3, step: 100, loss: 0.6630284786224365, acc:0.578125\n",
            "epoch: 3, step: 200, loss: 0.696617841720581, acc:0.53125\n",
            "epoch: 3, step: 300, loss: 0.7630640268325806, acc:0.5625\n",
            "epoch: 4, step: 0, loss: 0.49356481432914734, acc:0.796875\n",
            "epoch: 4, step: 100, loss: 0.5853723287582397, acc:0.734375\n",
            "epoch: 4, step: 200, loss: 0.42977458238601685, acc:0.765625\n",
            "epoch: 4, step: 300, loss: 0.753055214881897, acc:0.640625\n",
            "epoch: 5, step: 0, loss: 0.48700326681137085, acc:0.84375\n",
            "epoch: 5, step: 100, loss: 0.4442210793495178, acc:0.796875\n",
            "epoch: 5, step: 200, loss: 0.35492458939552307, acc:0.875\n",
            "epoch: 5, step: 300, loss: 0.4540897607803345, acc:0.765625\n",
            "epoch: 6, step: 0, loss: 0.5202851295471191, acc:0.765625\n",
            "epoch: 6, step: 100, loss: 0.5465481877326965, acc:0.71875\n",
            "epoch: 6, step: 200, loss: 0.36656463146209717, acc:0.84375\n",
            "epoch: 6, step: 300, loss: 0.40411120653152466, acc:0.800000011920929\n",
            "epoch: 7, step: 0, loss: 0.38006412982940674, acc:0.78125\n",
            "epoch: 7, step: 100, loss: 0.3438127636909485, acc:0.875\n",
            "epoch: 7, step: 200, loss: 0.3370388150215149, acc:0.8125\n",
            "epoch: 7, step: 300, loss: 0.37696921825408936, acc:0.828125\n",
            "epoch: 8, step: 0, loss: 0.2766188383102417, acc:0.859375\n",
            "epoch: 8, step: 100, loss: 0.2925263047218323, acc:0.875\n",
            "epoch: 8, step: 200, loss: 0.24380049109458923, acc:0.921875\n",
            "epoch: 8, step: 300, loss: 0.33437132835388184, acc:0.859375\n",
            "epoch: 9, step: 0, loss: 0.22632789611816406, acc:0.90625\n",
            "epoch: 9, step: 100, loss: 0.4952205419540405, acc:0.796875\n",
            "epoch: 9, step: 200, loss: 0.3096369206905365, acc:0.9375\n",
            "epoch: 9, step: 300, loss: 0.1888168901205063, acc:0.9375\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lr_dCZ1nmpHa",
        "colab_type": "code",
        "outputId": "2a8c307d-ecbe-43d8-debd-9f72c8744f65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "avg_acc = []\n",
        "with torch.no_grad():\n",
        "  for batch in test_iterator:\n",
        "\n",
        "    pred = network(batch.text).squeeze()\n",
        "\n",
        "    loss = criteon(pred, batch.label)\n",
        "\n",
        "    acc = binary_acc(pred, batch.label).item()\n",
        "    avg_acc.append(acc)\n",
        "    \n",
        "    # print(\"test loss: {}, test acc: {}\".format(loss.item(), acc))\n",
        "\n",
        "acc_avg = np.array(avg_acc).mean()\n",
        "\n",
        "print(\"Average acc: \", acc_avg)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average acc:  0.8639226342406114\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_XokVtDv45mF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}